<?xml version='1.0' encoding="utf-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html lang="en" xml:lang="en" xmlns="http://www.w3.org/1999/xhtml">
  <head>
    <link href="cpip.css" rel="stylesheet" type="text/css" />
    <title>File: /Users/paulross/dev/linux/linux-3.13/arch/x86/include/asm/spinlock.h</title>
  </head>
  <body>
    <h1>File: /Users/paulross/dev/linux/linux-3.13/arch/x86/include/asm/spinlock.h</h1>
    <p>Green shading in the line number column
means the source is part of the translation unit, red means it is conditionally excluded.
Highlighted line numbers link to the translation unit page. Highlighted macros link to
the macro page.</p>
    <pre><a name="1" /><span class="True">       1:</span> <span class="f">#</span><span class="n">ifndef</span> <a href="cpu.c_macros_noref.html#_X0FTTV9YODZfU1BJTkxPQ0tfSF8w"><span class="b">_ASM_X86_SPINLOCK_H</span></a>
<a name="2" /><span class="True">       2:</span> <span class="f">#</span><span class="n">define</span> <a href="cpu.c_macros_noref.html#_X0FTTV9YODZfU1BJTkxPQ0tfSF8w"><span class="b">_ASM_X86_SPINLOCK_H</span></a>
<a name="3" /><span class="True">       3:</span> 
<a name="4" /><span class="True">       4:</span> <span class="f">#</span><span class="n">include</span> <span class="f">&lt;</span><span class="b">linux</span><span class="f">/</span><span class="b">jump_label</span><span class="f">.</span><span class="b">h</span><span class="f">&gt;</span>
<a name="5" /><span class="True">       5:</span> <span class="f">#</span><span class="n">include</span> <span class="f">&lt;</span><span class="b">linux</span><span class="f">/</span><span class="b">atomic</span><span class="f">.</span><span class="b">h</span><span class="f">&gt;</span>
<a name="6" /><span class="True">       6:</span> <span class="f">#</span><span class="n">include</span> <span class="f">&lt;</span><span class="m">asm</span><span class="f">/</span><span class="b">page</span><span class="f">.</span><span class="b">h</span><span class="f">&gt;</span>
<a name="7" /><span class="True">       7:</span> <span class="f">#</span><span class="n">include</span> <span class="f">&lt;</span><span class="m">asm</span><span class="f">/</span><span class="b">processor</span><span class="f">.</span><span class="b">h</span><span class="f">&gt;</span>
<a name="8" /><span class="True">       8:</span> <span class="f">#</span><span class="n">include</span> <span class="f">&lt;</span><span class="b">linux</span><span class="f">/</span><span class="b">compiler</span><span class="f">.</span><span class="b">h</span><span class="f">&gt;</span>
<a name="9" /><span class="True">       9:</span> <span class="f">#</span><span class="n">include</span> <span class="f">&lt;</span><span class="m">asm</span><span class="f">/</span><span class="b">paravirt</span><span class="f">.</span><span class="b">h</span><span class="f">&gt;</span>
<a name="10" /><span class="True">      10:</span> <span class="f">#</span><span class="n">include</span> <span class="f">&lt;</span><span class="m">asm</span><span class="f">/</span><span class="b">bitops</span><span class="f">.</span><span class="b">h</span><span class="f">&gt;</span>
<a name="11" /><span class="True">      11:</span> 
<a name="12" /><span class="True">      12:</span> <span class="k">/*</span>
<a name="13" /><span class="True">      13:</span> <span class="k"> * Your basic SMP spinlocks, allowing only a single CPU anywhere</span>
<a name="14" /><span class="True">      14:</span> <span class="k"> *</span>
<a name="15" /><span class="True">      15:</span> <span class="k"> * Simple spin lock operations.  There are two variants, one clears IRQ&apos;s</span>
<a name="16" /><span class="True">      16:</span> <span class="k"> * on the local processor, one does not.</span>
<a name="17" /><span class="True">      17:</span> <span class="k"> *</span>
<a name="18" /><span class="True">      18:</span> <span class="k"> * These are fair FIFO ticket locks, which support up to 2^16 CPUs.</span>
<a name="19" /><span class="True">      19:</span> <span class="k"> *</span>
<a name="20" /><span class="True">      20:</span> <span class="k"> * (the type definitions are in asm/spinlock_types.h)</span>
<a name="21" /><span class="True">      21:</span> <span class="k"> */</span>
<a name="22" /><span class="True">      22:</span> 
<a name="23" /><span class="False">      23:</span> <span class="f">#</span><span class="n">ifdef</span> <span class="b">CONFIG_X86_32</span>
<a name="24" /><span class="False">      24:</span> <span class="f">#</span> <span class="n">define</span> <a href="cpu.c_macros_ref.html#_TE9DS19QVFJfUkVHXzA_"><span class="b">LOCK_PTR_REG</span></a> <span class="e">&quot;a&quot;</span>
<a name="25" /><span class="True">      25:</span> <span class="f">#</span><span class="n">else</span>
<a name="26" /><span class="True">      26:</span> <span class="f">#</span> <span class="n">define</span> <a href="cpu.c_macros_ref.html#_TE9DS19QVFJfUkVHXzA_"><span class="b">LOCK_PTR_REG</span></a> <span class="e">&quot;D&quot;</span>
<a name="27" /><span class="True">      27:</span> <span class="f">#</span><span class="n">endif</span>
<a name="28" /><span class="True">      28:</span> 
<a name="29" /><span class="False">      29:</span> <span class="f">#</span><span class="n">if</span> <span class="b">defined</span><span class="f">(</span><span class="b">CONFIG_X86_32</span><span class="f">)</span> <span class="f">&amp;&amp;</span> \
<a name="30" /><span class="False">      30:</span>     <span class="f">(</span><span class="b">defined</span><span class="f">(</span><span class="b">CONFIG_X86_OOSTORE</span><span class="f">)</span> <span class="f">||</span> <span class="b">defined</span><span class="f">(</span><span class="b">CONFIG_X86_PPRO_FENCE</span><span class="f">)</span><span class="f">)</span>
<a name="31" /><span class="False">      31:</span> <span class="k">/*</span>
<a name="32" /><span class="False">      32:</span> <span class="k"> * On PPro SMP or if we are using OOSTORE, we use a locked operation to unlock</span>
<a name="33" /><span class="False">      33:</span> <span class="k"> * (PPro errata 66, 92)</span>
<a name="34" /><span class="False">      34:</span> <span class="k"> */</span>
<a name="35" /><span class="False">      35:</span> <span class="f">#</span> <span class="n">define</span> <a href="cpu.c_macros_ref.html#_VU5MT0NLX0xPQ0tfUFJFRklYXzA_"><span class="b">UNLOCK_LOCK_PREFIX</span></a> <a href="cpu.c_macros_ref.html#_TE9DS19QUkVGSVhfMA__"><span class="b">LOCK_PREFIX</span></a>
<a name="36" /><span class="True">      36:</span> <span class="f">#</span><span class="n">else</span>
<a name="37" /><span class="True">      37:</span> <span class="f">#</span> <span class="n">define</span> <a href="cpu.c_macros_ref.html#_VU5MT0NLX0xPQ0tfUFJFRklYXzA_"><span class="b">UNLOCK_LOCK_PREFIX</span></a>
<a name="38" /><span class="True">      38:</span> <span class="f">#</span><span class="n">endif</span>
<a name="39" /><span class="True">      39:</span> 
<a name="40" /><span class="True">      40:</span> <span class="k">/* How long a lock should spin before we consider blocking */</span>
<a name="41" /><span class="True">      41:</span> <span class="f">#</span><span class="n">define</span> <a href="cpu.c_macros_ref.html#_U1BJTl9USFJFU0hPTERfMA__"><span class="b">SPIN_THRESHOLD</span></a>    <span class="f">(</span><span class="c">1</span> <span class="f">&lt;&lt;</span> <span class="c">15</span><span class="f">)</span>
<a name="42" /><span class="True">      42:</span> 
<a name="43" /><span class="True">      43:</span> <span class="m">extern</span> <span class="m">struct</span> <span class="b">static_key</span> <span class="b">paravirt_ticketlocks_enabled</span><span class="f">;</span>
<a name="44" /><span class="True">      44:</span> <span class="m">static</span> <a href="cpu.c_macros_ref.html#_X19hbHdheXNfaW5saW5lXzA_"><span class="b">__always_inline</span></a> <span class="m">bool</span> <span class="b">static_key_false</span><span class="f">(</span><span class="m">struct</span> <span class="b">static_key</span> <span class="f">*</span><span class="b">key</span><span class="f">)</span><span class="f">;</span>
<a name="45" /><span class="True">      45:</span> 
<a name="46" /><span class="False">      46:</span> <span class="f">#</span><span class="n">ifdef</span> <span class="b">CONFIG_PARAVIRT_SPINLOCKS</span>
<a name="47" /><span class="False">      47:</span> 
<a name="48" /><span class="False">      48:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">void</span> <span class="b">__ticket_enter_slowpath</span><span class="f">(</span><span class="b">arch_spinlock_t</span> <span class="f">*</span><span class="b">lock</span><span class="f">)</span>
<a name="49" /><span class="False">      49:</span> <span class="f">{</span>
<a name="50" /><span class="False">      50:</span>     <span class="b">set_bit</span><span class="f">(</span><span class="c">0</span><span class="f">,</span> <span class="f">(</span><span class="m">volatile</span> <span class="m">unsigned</span> <span class="m">long</span> <span class="f">*</span><span class="f">)</span><span class="f">&amp;</span><span class="b">lock</span><span class="f">-&gt;</span><span class="b">tickets</span><span class="f">.</span><span class="b">tail</span><span class="f">)</span><span class="f">;</span>
<a name="51" /><span class="False">      51:</span> <span class="f">}</span>
<a name="52" /><span class="False">      52:</span> 
<a name="53" /><span class="True">      53:</span> <span class="f">#</span><span class="n">else</span>  <span class="k">/* !CONFIG_PARAVIRT_SPINLOCKS */</span>
<a name="54" /><span class="True">      54:</span> <span class="m">static</span> <a href="cpu.c_macros_ref.html#_X19hbHdheXNfaW5saW5lXzA_"><span class="b">__always_inline</span></a> <span class="m">void</span> <span class="b">__ticket_lock_spinning</span><span class="f">(</span><span class="b">arch_spinlock_t</span> <span class="f">*</span><span class="b">lock</span><span class="f">,</span>
<a name="55" /><span class="True">      55:</span>                             <span class="b">__ticket_t</span> <span class="b">ticket</span><span class="f">)</span>
<a name="56" /><span class="True">      56:</span> <span class="f">{</span>
<a name="57" /><span class="True">      57:</span> <span class="f">}</span>
<a name="58" /><span class="True">      58:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">void</span> <span class="b">__ticket_unlock_kick</span><span class="f">(</span><span class="b">arch_spinlock_t</span> <span class="f">*</span><span class="b">lock</span><span class="f">,</span>
<a name="59" /><span class="True">      59:</span>                             <span class="b">__ticket_t</span> <span class="b">ticket</span><span class="f">)</span>
<a name="60" /><span class="True">      60:</span> <span class="f">{</span>
<a name="61" /><span class="True">      61:</span> <span class="f">}</span>
<a name="62" /><span class="True">      62:</span> 
<a name="63" /><span class="True">      63:</span> <span class="f">#</span><span class="n">endif</span> <span class="k">/* CONFIG_PARAVIRT_SPINLOCKS */</span>
<a name="64" /><span class="True">      64:</span> 
<a name="65" /><span class="True">      65:</span> <span class="m">static</span> <a href="cpu.c_macros_ref.html#_X19hbHdheXNfaW5saW5lXzA_"><span class="b">__always_inline</span></a> <span class="m">int</span> <span class="b">arch_spin_value_unlocked</span><span class="f">(</span><span class="b">arch_spinlock_t</span> <span class="b">lock</span><span class="f">)</span>
<a name="66" /><span class="True">      66:</span> <span class="f">{</span>
<a name="67" /><span class="True">      67:</span>     <span class="m">return</span> <span class="b">lock</span><span class="f">.</span><span class="b">tickets</span><span class="f">.</span><span class="b">head</span> <span class="f">==</span> <span class="b">lock</span><span class="f">.</span><span class="b">tickets</span><span class="f">.</span><span class="b">tail</span><span class="f">;</span>
<a name="68" /><span class="True">      68:</span> <span class="f">}</span>
<a name="69" /><span class="True">      69:</span> 
<a name="70" /><span class="True">      70:</span> <span class="k">/*</span>
<a name="71" /><span class="True">      71:</span> <span class="k"> * Ticket locks are conceptually two parts, one indicating the current head of</span>
<a name="72" /><span class="True">      72:</span> <span class="k"> * the queue, and the other indicating the current tail. The lock is acquired</span>
<a name="73" /><span class="True">      73:</span> <span class="k"> * by atomically noting the tail and incrementing it by one (thus adding</span>
<a name="74" /><span class="True">      74:</span> <span class="k"> * ourself to the queue and noting our position), then waiting until the head</span>
<a name="75" /><span class="True">      75:</span> <span class="k"> * becomes equal to the the initial value of the tail.</span>
<a name="76" /><span class="True">      76:</span> <span class="k"> *</span>
<a name="77" /><span class="True">      77:</span> <span class="k"> * We use an xadd covering *both* parts of the lock, to increment the tail and</span>
<a name="78" /><span class="True">      78:</span> <span class="k"> * also load the position of the head, which takes care of memory ordering</span>
<a name="79" /><span class="True">      79:</span> <span class="k"> * issues and should be optimal for the uncontended case. Note the tail must be</span>
<a name="80" /><span class="True">      80:</span> <span class="k"> * in the high part, because a wide xadd increment of the low part would carry</span>
<a name="81" /><span class="True">      81:</span> <span class="k"> * up and contaminate the high part.</span>
<a name="82" /><span class="True">      82:</span> <span class="k"> */</span>
<a name="83" /><span class="True">      83:</span> <span class="m">static</span> <a href="cpu.c_macros_ref.html#_X19hbHdheXNfaW5saW5lXzA_"><span class="b">__always_inline</span></a> <span class="m">void</span> <span class="b">arch_spin_lock</span><span class="f">(</span><span class="b">arch_spinlock_t</span> <span class="f">*</span><span class="b">lock</span><span class="f">)</span>
<a name="84" /><span class="True">      84:</span> <span class="f">{</span>
<a name="85" /><span class="True">      85:</span>     <span class="m">register</span> <span class="m">struct</span> <span class="b">__raw_tickets</span> <span class="b">inc</span> <span class="f">=</span> <span class="f">{</span> <span class="f">.</span><span class="b">tail</span> <span class="f">=</span> <a href="cpu.c_macros_ref.html#_VElDS0VUX0xPQ0tfSU5DXzA_"><span class="b">TICKET_LOCK_INC</span></a> <span class="f">}</span><span class="f">;</span>
<a name="86" /><span class="True">      86:</span> 
<a name="87" /><span class="True">      87:</span>     <span class="b">inc</span> <span class="f">=</span> <a href="cpu.c_macros_ref.html#_eGFkZF8w"><span class="b">xadd</span></a><span class="f">(</span><span class="f">&amp;</span><span class="b">lock</span><span class="f">-&gt;</span><span class="b">tickets</span><span class="f">,</span> <span class="b">inc</span><span class="f">)</span><span class="f">;</span>
<a name="88" /><span class="True">      88:</span>     <span class="m">if</span> <span class="f">(</span><a href="cpu.c_macros_ref.html#_bGlrZWx5XzA_"><span class="b">likely</span></a><span class="f">(</span><span class="b">inc</span><span class="f">.</span><span class="b">head</span> <span class="f">==</span> <span class="b">inc</span><span class="f">.</span><span class="b">tail</span><span class="f">)</span><span class="f">)</span>
<a name="89" /><span class="True">      89:</span>         <span class="m">goto</span> <span class="b">out</span><span class="f">;</span>
<a name="90" /><span class="True">      90:</span> 
<a name="91" /><span class="True">      91:</span>     <span class="b">inc</span><span class="f">.</span><span class="b">tail</span> <span class="f">&amp;=</span> <span class="f">~</span><a href="cpu.c_macros_ref.html#_VElDS0VUX1NMT1dQQVRIX0ZMQUdfMA__"><span class="b">TICKET_SLOWPATH_FLAG</span></a><span class="f">;</span>
<a name="92" /><span class="True">      92:</span>     <span class="m">for</span> <span class="f">(</span><span class="f">;</span><span class="f">;</span><span class="f">)</span> <span class="f">{</span>
<a name="93" /><span class="True">      93:</span>         <span class="m">unsigned</span> <span class="b">count</span> <span class="f">=</span> <a href="cpu.c_macros_ref.html#_U1BJTl9USFJFU0hPTERfMA__"><span class="b">SPIN_THRESHOLD</span></a><span class="f">;</span>
<a name="94" /><span class="True">      94:</span> 
<a name="95" /><span class="True">      95:</span>         <span class="m">do</span> <span class="f">{</span>
<a name="96" /><span class="True">      96:</span>             <span class="m">if</span> <span class="f">(</span><a href="cpu.c_macros_ref.html#_QUNDRVNTX09OQ0VfMA__"><span class="b">ACCESS_ONCE</span></a><span class="f">(</span><span class="b">lock</span><span class="f">-&gt;</span><span class="b">tickets</span><span class="f">.</span><span class="b">head</span><span class="f">)</span> <span class="f">==</span> <span class="b">inc</span><span class="f">.</span><span class="b">tail</span><span class="f">)</span>
<a name="97" /><span class="True">      97:</span>                 <span class="m">goto</span> <span class="b">out</span><span class="f">;</span>
<a name="98" /><span class="True">      98:</span>             <span class="b">cpu_relax</span><span class="f">(</span><span class="f">)</span><span class="f">;</span>
<a name="99" /><span class="True">      99:</span>         <span class="f">}</span> <span class="m">while</span> <span class="f">(</span><span class="f">--</span><span class="b">count</span><span class="f">)</span><span class="f">;</span>
<a name="100" /><span class="True">     100:</span>         <span class="b">__ticket_lock_spinning</span><span class="f">(</span><span class="b">lock</span><span class="f">,</span> <span class="b">inc</span><span class="f">.</span><span class="b">tail</span><span class="f">)</span><span class="f">;</span>
<a name="101" /><span class="True">     101:</span>     <span class="f">}</span>
<a name="102" /><span class="True">     102:</span> <span class="b">out</span><span class="f">:</span>    <a href="cpu.c_macros_ref.html#_YmFycmllcl8w"><span class="b">barrier</span></a><span class="f">(</span><span class="f">)</span><span class="f">;</span>    <span class="k">/* make sure nothing creeps before the lock is taken */</span>
<a name="103" /><span class="True">     103:</span> <span class="f">}</span>
<a name="104" /><span class="True">     104:</span> 
<a name="105" /><span class="True">     105:</span> <span class="m">static</span> <a href="cpu.c_macros_ref.html#_X19hbHdheXNfaW5saW5lXzA_"><span class="b">__always_inline</span></a> <span class="m">int</span> <span class="b">arch_spin_trylock</span><span class="f">(</span><span class="b">arch_spinlock_t</span> <span class="f">*</span><span class="b">lock</span><span class="f">)</span>
<a name="106" /><span class="True">     106:</span> <span class="f">{</span>
<a name="107" /><span class="True">     107:</span>     <span class="b">arch_spinlock_t</span> <span class="b">old</span><span class="f">,</span> <span class="m">new</span><span class="f">;</span>
<a name="108" /><span class="True">     108:</span> 
<a name="109" /><span class="True">     109:</span>     <span class="b">old</span><span class="f">.</span><span class="b">tickets</span> <span class="f">=</span> <a href="cpu.c_macros_ref.html#_QUNDRVNTX09OQ0VfMA__"><span class="b">ACCESS_ONCE</span></a><span class="f">(</span><span class="b">lock</span><span class="f">-&gt;</span><span class="b">tickets</span><span class="f">)</span><span class="f">;</span>
<a name="110" /><span class="True">     110:</span>     <span class="m">if</span> <span class="f">(</span><span class="b">old</span><span class="f">.</span><span class="b">tickets</span><span class="f">.</span><span class="b">head</span> <span class="f">!=</span> <span class="f">(</span><span class="b">old</span><span class="f">.</span><span class="b">tickets</span><span class="f">.</span><span class="b">tail</span> <span class="f">&amp;</span> <span class="f">~</span><a href="cpu.c_macros_ref.html#_VElDS0VUX1NMT1dQQVRIX0ZMQUdfMA__"><span class="b">TICKET_SLOWPATH_FLAG</span></a><span class="f">)</span><span class="f">)</span>
<a name="111" /><span class="True">     111:</span>         <span class="m">return</span> <span class="c">0</span><span class="f">;</span>
<a name="112" /><span class="True">     112:</span> 
<a name="113" /><span class="True">     113:</span>     <span class="m">new</span><span class="f">.</span><span class="b">head_tail</span> <span class="f">=</span> <span class="b">old</span><span class="f">.</span><span class="b">head_tail</span> <span class="f">+</span> <span class="f">(</span><a href="cpu.c_macros_ref.html#_VElDS0VUX0xPQ0tfSU5DXzA_"><span class="b">TICKET_LOCK_INC</span></a> <span class="f">&lt;&lt;</span> <a href="cpu.c_macros_ref.html#_VElDS0VUX1NISUZUXzA_"><span class="b">TICKET_SHIFT</span></a><span class="f">)</span><span class="f">;</span>
<a name="114" /><span class="True">     114:</span> 
<a name="115" /><span class="True">     115:</span>     <span class="k">/* cmpxchg is a full barrier, so nothing can move before it */</span>
<a name="116" /><span class="True">     116:</span>     <span class="m">return</span> <a href="cpu.c_macros_ref.html#_Y21weGNoZ18w"><span class="b">cmpxchg</span></a><span class="f">(</span><span class="f">&amp;</span><span class="b">lock</span><span class="f">-&gt;</span><span class="b">head_tail</span><span class="f">,</span> <span class="b">old</span><span class="f">.</span><span class="b">head_tail</span><span class="f">,</span> <span class="m">new</span><span class="f">.</span><span class="b">head_tail</span><span class="f">)</span> <span class="f">==</span> <span class="b">old</span><span class="f">.</span><span class="b">head_tail</span><span class="f">;</span>
<a name="117" /><span class="True">     117:</span> <span class="f">}</span>
<a name="118" /><span class="True">     118:</span> 
<a name="119" /><span class="True">     119:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">void</span> <span class="b">__ticket_unlock_slowpath</span><span class="f">(</span><span class="b">arch_spinlock_t</span> <span class="f">*</span><span class="b">lock</span><span class="f">,</span>
<a name="120" /><span class="True">     120:</span>                         <span class="b">arch_spinlock_t</span> <span class="b">old</span><span class="f">)</span>
<a name="121" /><span class="True">     121:</span> <span class="f">{</span>
<a name="122" /><span class="True">     122:</span>     <span class="b">arch_spinlock_t</span> <span class="m">new</span><span class="f">;</span>
<a name="123" /><span class="True">     123:</span> 
<a name="124" /><span class="True">     124:</span>     <a href="cpu.c_macros_ref.html#_QlVJTERfQlVHX09OXzA_"><span class="b">BUILD_BUG_ON</span></a><span class="f">(</span><span class="f">(</span><span class="f">(</span><span class="b">__ticket_t</span><span class="f">)</span><a href="cpu.c_macros_ref.html#_TlJfQ1BVU18w"><span class="b">NR_CPUS</span></a><span class="f">)</span> <span class="f">!=</span> <a href="cpu.c_macros_ref.html#_TlJfQ1BVU18w"><span class="b">NR_CPUS</span></a><span class="f">)</span><span class="f">;</span>
<a name="125" /><span class="True">     125:</span> 
<a name="126" /><span class="True">     126:</span>     <span class="k">/* Perform the unlock on the &quot;before&quot; copy */</span>
<a name="127" /><span class="True">     127:</span>     <span class="b">old</span><span class="f">.</span><span class="b">tickets</span><span class="f">.</span><span class="b">head</span> <span class="f">+=</span> <a href="cpu.c_macros_ref.html#_VElDS0VUX0xPQ0tfSU5DXzA_"><span class="b">TICKET_LOCK_INC</span></a><span class="f">;</span>
<a name="128" /><span class="True">     128:</span> 
<a name="129" /><span class="True">     129:</span>     <span class="k">/* Clear the slowpath flag */</span>
<a name="130" /><span class="True">     130:</span>     <span class="m">new</span><span class="f">.</span><span class="b">head_tail</span> <span class="f">=</span> <span class="b">old</span><span class="f">.</span><span class="b">head_tail</span> <span class="f">&amp;</span> <span class="f">~</span><span class="f">(</span><a href="cpu.c_macros_ref.html#_VElDS0VUX1NMT1dQQVRIX0ZMQUdfMA__"><span class="b">TICKET_SLOWPATH_FLAG</span></a> <span class="f">&lt;&lt;</span> <a href="cpu.c_macros_ref.html#_VElDS0VUX1NISUZUXzA_"><span class="b">TICKET_SHIFT</span></a><span class="f">)</span><span class="f">;</span>
<a name="131" /><span class="True">     131:</span> 
<a name="132" /><span class="True">     132:</span>     <span class="k">/*</span>
<a name="133" /><span class="True">     133:</span> <span class="k">     * If the lock is uncontended, clear the flag - use cmpxchg in</span>
<a name="134" /><span class="True">     134:</span> <span class="k">     * case it changes behind our back though.</span>
<a name="135" /><span class="True">     135:</span> <span class="k">     */</span>
<a name="136" /><span class="True">     136:</span>     <span class="m">if</span> <span class="f">(</span><span class="m">new</span><span class="f">.</span><span class="b">tickets</span><span class="f">.</span><span class="b">head</span> <span class="f">!=</span> <span class="m">new</span><span class="f">.</span><span class="b">tickets</span><span class="f">.</span><span class="b">tail</span> <span class="f">||</span>
<a name="137" /><span class="True">     137:</span>         <a href="cpu.c_macros_ref.html#_Y21weGNoZ18w"><span class="b">cmpxchg</span></a><span class="f">(</span><span class="f">&amp;</span><span class="b">lock</span><span class="f">-&gt;</span><span class="b">head_tail</span><span class="f">,</span> <span class="b">old</span><span class="f">.</span><span class="b">head_tail</span><span class="f">,</span>
<a name="138" /><span class="True">     138:</span>                     <span class="m">new</span><span class="f">.</span><span class="b">head_tail</span><span class="f">)</span> <span class="f">!=</span> <span class="b">old</span><span class="f">.</span><span class="b">head_tail</span><span class="f">)</span> <span class="f">{</span>
<a name="139" /><span class="True">     139:</span>         <span class="k">/*</span>
<a name="140" /><span class="True">     140:</span> <span class="k">         * Lock still has someone queued for it, so wake up an</span>
<a name="141" /><span class="True">     141:</span> <span class="k">         * appropriate waiter.</span>
<a name="142" /><span class="True">     142:</span> <span class="k">         */</span>
<a name="143" /><span class="True">     143:</span>         <span class="b">__ticket_unlock_kick</span><span class="f">(</span><span class="b">lock</span><span class="f">,</span> <span class="b">old</span><span class="f">.</span><span class="b">tickets</span><span class="f">.</span><span class="b">head</span><span class="f">)</span><span class="f">;</span>
<a name="144" /><span class="True">     144:</span>     <span class="f">}</span>
<a name="145" /><span class="True">     145:</span> <span class="f">}</span>
<a name="146" /><span class="True">     146:</span> 
<a name="147" /><span class="True">     147:</span> <span class="m">static</span> <a href="cpu.c_macros_ref.html#_X19hbHdheXNfaW5saW5lXzA_"><span class="b">__always_inline</span></a> <span class="m">void</span> <span class="b">arch_spin_unlock</span><span class="f">(</span><span class="b">arch_spinlock_t</span> <span class="f">*</span><span class="b">lock</span><span class="f">)</span>
<a name="148" /><span class="True">     148:</span> <span class="f">{</span>
<a name="149" /><span class="True">     149:</span>     <span class="m">if</span> <span class="f">(</span><a href="cpu.c_macros_ref.html#_VElDS0VUX1NMT1dQQVRIX0ZMQUdfMA__"><span class="b">TICKET_SLOWPATH_FLAG</span></a> <span class="f">&amp;&amp;</span>
<a name="150" /><span class="True">     150:</span>         <span class="b">static_key_false</span><span class="f">(</span><span class="f">&amp;</span><span class="b">paravirt_ticketlocks_enabled</span><span class="f">)</span><span class="f">)</span> <span class="f">{</span>
<a name="151" /><span class="True">     151:</span>         <span class="b">arch_spinlock_t</span> <span class="b">prev</span><span class="f">;</span>
<a name="152" /><span class="True">     152:</span> 
<a name="153" /><span class="True">     153:</span>         <span class="b">prev</span> <span class="f">=</span> <span class="f">*</span><span class="b">lock</span><span class="f">;</span>
<a name="154" /><span class="True">     154:</span>         <a href="cpu.c_macros_ref.html#_YWRkX3NtcF8w"><span class="b">add_smp</span></a><span class="f">(</span><span class="f">&amp;</span><span class="b">lock</span><span class="f">-&gt;</span><span class="b">tickets</span><span class="f">.</span><span class="b">head</span><span class="f">,</span> <a href="cpu.c_macros_ref.html#_VElDS0VUX0xPQ0tfSU5DXzA_"><span class="b">TICKET_LOCK_INC</span></a><span class="f">)</span><span class="f">;</span>
<a name="155" /><span class="True">     155:</span> 
<a name="156" /><span class="True">     156:</span>         <span class="k">/* add_smp() is a full mb() */</span>
<a name="157" /><span class="True">     157:</span> 
<a name="158" /><span class="True">     158:</span>         <span class="m">if</span> <span class="f">(</span><a href="cpu.c_macros_ref.html#_dW5saWtlbHlfMA__"><span class="b">unlikely</span></a><span class="f">(</span><span class="b">lock</span><span class="f">-&gt;</span><span class="b">tickets</span><span class="f">.</span><span class="b">tail</span> <span class="f">&amp;</span> <a href="cpu.c_macros_ref.html#_VElDS0VUX1NMT1dQQVRIX0ZMQUdfMA__"><span class="b">TICKET_SLOWPATH_FLAG</span></a><span class="f">)</span><span class="f">)</span>
<a name="159" /><span class="True">     159:</span>             <span class="b">__ticket_unlock_slowpath</span><span class="f">(</span><span class="b">lock</span><span class="f">,</span> <span class="b">prev</span><span class="f">)</span><span class="f">;</span>
<a name="160" /><span class="True">     160:</span>     <span class="f">}</span> <span class="m">else</span>
<a name="161" /><span class="True">     161:</span>         <a href="cpu.c_macros_ref.html#_X19hZGRfMA__"><span class="b">__add</span></a><span class="f">(</span><span class="f">&amp;</span><span class="b">lock</span><span class="f">-&gt;</span><span class="b">tickets</span><span class="f">.</span><span class="b">head</span><span class="f">,</span> <a href="cpu.c_macros_ref.html#_VElDS0VUX0xPQ0tfSU5DXzA_"><span class="b">TICKET_LOCK_INC</span></a><span class="f">,</span> <a href="cpu.c_macros_ref.html#_VU5MT0NLX0xPQ0tfUFJFRklYXzA_"><span class="b">UNLOCK_LOCK_PREFIX</span></a><span class="f">)</span><span class="f">;</span>
<a name="162" /><span class="True">     162:</span> <span class="f">}</span>
<a name="163" /><span class="True">     163:</span> 
<a name="164" /><span class="True">     164:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">int</span> <span class="b">arch_spin_is_locked</span><span class="f">(</span><span class="b">arch_spinlock_t</span> <span class="f">*</span><span class="b">lock</span><span class="f">)</span>
<a name="165" /><span class="True">     165:</span> <span class="f">{</span>
<a name="166" /><span class="True">     166:</span>     <span class="m">struct</span> <span class="b">__raw_tickets</span> <span class="b">tmp</span> <span class="f">=</span> <a href="cpu.c_macros_ref.html#_QUNDRVNTX09OQ0VfMA__"><span class="b">ACCESS_ONCE</span></a><span class="f">(</span><span class="b">lock</span><span class="f">-&gt;</span><span class="b">tickets</span><span class="f">)</span><span class="f">;</span>
<a name="167" /><span class="True">     167:</span> 
<a name="168" /><span class="True">     168:</span>     <span class="m">return</span> <span class="b">tmp</span><span class="f">.</span><span class="b">tail</span> <span class="f">!=</span> <span class="b">tmp</span><span class="f">.</span><span class="b">head</span><span class="f">;</span>
<a name="169" /><span class="True">     169:</span> <span class="f">}</span>
<a name="170" /><span class="True">     170:</span> 
<a name="171" /><span class="True">     171:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">int</span> <a href="cpu.c_macros_ref.html#_YXJjaF9zcGluX2lzX2NvbnRlbmRlZF8w"><span class="b">arch_spin_is_contended</span></a><span class="f">(</span><span class="b">arch_spinlock_t</span> <span class="f">*</span><span class="b">lock</span><span class="f">)</span>
<a name="172" /><span class="True">     172:</span> <span class="f">{</span>
<a name="173" /><span class="True">     173:</span>     <span class="m">struct</span> <span class="b">__raw_tickets</span> <span class="b">tmp</span> <span class="f">=</span> <a href="cpu.c_macros_ref.html#_QUNDRVNTX09OQ0VfMA__"><span class="b">ACCESS_ONCE</span></a><span class="f">(</span><span class="b">lock</span><span class="f">-&gt;</span><span class="b">tickets</span><span class="f">)</span><span class="f">;</span>
<a name="174" /><span class="True">     174:</span> 
<a name="175" /><span class="True">     175:</span>     <span class="m">return</span> <span class="f">(</span><span class="b">__ticket_t</span><span class="f">)</span><span class="f">(</span><span class="b">tmp</span><span class="f">.</span><span class="b">tail</span> <span class="f">-</span> <span class="b">tmp</span><span class="f">.</span><span class="b">head</span><span class="f">)</span> <span class="f">&gt;</span> <a href="cpu.c_macros_ref.html#_VElDS0VUX0xPQ0tfSU5DXzA_"><span class="b">TICKET_LOCK_INC</span></a><span class="f">;</span>
<a name="176" /><span class="True">     176:</span> <span class="f">}</span>
<a name="177" /><span class="True">     177:</span> <span class="f">#</span><span class="n">define</span> <a href="cpu.c_macros_ref.html#_YXJjaF9zcGluX2lzX2NvbnRlbmRlZF8w"><span class="b">arch_spin_is_contended</span></a>    <a href="cpu.c_macros_ref.html#_YXJjaF9zcGluX2lzX2NvbnRlbmRlZF8w"><span class="b">arch_spin_is_contended</span></a>
<a name="178" /><span class="True">     178:</span> 
<a name="179" /><span class="True">     179:</span> <span class="m">static</span> <a href="cpu.c_macros_ref.html#_X19hbHdheXNfaW5saW5lXzA_"><span class="b">__always_inline</span></a> <span class="m">void</span> <span class="b">arch_spin_lock_flags</span><span class="f">(</span><span class="b">arch_spinlock_t</span> <span class="f">*</span><span class="b">lock</span><span class="f">,</span>
<a name="180" /><span class="True">     180:</span>                           <span class="m">unsigned</span> <span class="m">long</span> <span class="b">flags</span><span class="f">)</span>
<a name="181" /><span class="True">     181:</span> <span class="f">{</span>
<a name="182" /><span class="True">     182:</span>     <span class="b">arch_spin_lock</span><span class="f">(</span><span class="b">lock</span><span class="f">)</span><span class="f">;</span>
<a name="183" /><span class="True">     183:</span> <span class="f">}</span>
<a name="184" /><span class="True">     184:</span> 
<a name="185" /><span class="True">     185:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">void</span> <span class="b">arch_spin_unlock_wait</span><span class="f">(</span><span class="b">arch_spinlock_t</span> <span class="f">*</span><span class="b">lock</span><span class="f">)</span>
<a name="186" /><span class="True">     186:</span> <span class="f">{</span>
<a name="187" /><span class="True">     187:</span>     <span class="m">while</span> <span class="f">(</span><span class="b">arch_spin_is_locked</span><span class="f">(</span><span class="b">lock</span><span class="f">)</span><span class="f">)</span>
<a name="188" /><span class="True">     188:</span>         <span class="b">cpu_relax</span><span class="f">(</span><span class="f">)</span><span class="f">;</span>
<a name="189" /><span class="True">     189:</span> <span class="f">}</span>
<a name="190" /><span class="True">     190:</span> 
<a name="191" /><span class="True">     191:</span> <span class="k">/*</span>
<a name="192" /><span class="True">     192:</span> <span class="k"> * Read-write spinlocks, allowing multiple readers</span>
<a name="193" /><span class="True">     193:</span> <span class="k"> * but only one writer.</span>
<a name="194" /><span class="True">     194:</span> <span class="k"> *</span>
<a name="195" /><span class="True">     195:</span> <span class="k"> * NOTE! it is quite common to have readers in interrupts</span>
<a name="196" /><span class="True">     196:</span> <span class="k"> * but no interrupt writers. For those circumstances we</span>
<a name="197" /><span class="True">     197:</span> <span class="k"> * can &quot;mix&quot; irq-safe locks - any writer needs to get a</span>
<a name="198" /><span class="True">     198:</span> <span class="k"> * irq-safe write-lock, but readers can get non-irqsafe</span>
<a name="199" /><span class="True">     199:</span> <span class="k"> * read-locks.</span>
<a name="200" /><span class="True">     200:</span> <span class="k"> *</span>
<a name="201" /><span class="True">     201:</span> <span class="k"> * On x86, we implement read-write locks as a 32-bit counter</span>
<a name="202" /><span class="True">     202:</span> <span class="k"> * with the high bit (sign) being the &quot;contended&quot; bit.</span>
<a name="203" /><span class="True">     203:</span> <span class="k"> */</span>
<a name="204" /><span class="True">     204:</span> 
<a name="205" /><span class="True">     205:</span> <span class="k">/**</span>
<a name="206" /><span class="True">     206:</span> <span class="k"> * read_can_lock - would read_trylock() succeed?</span>
<a name="207" /><span class="True">     207:</span> <span class="k"> * @lock: the rwlock in question.</span>
<a name="208" /><span class="True">     208:</span> <span class="k"> */</span>
<a name="209" /><span class="True">     209:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">int</span> <span class="b">arch_read_can_lock</span><span class="f">(</span><span class="b">arch_rwlock_t</span> <span class="f">*</span><span class="b">lock</span><span class="f">)</span>
<a name="210" /><span class="True">     210:</span> <span class="f">{</span>
<a name="211" /><span class="True">     211:</span>     <span class="m">return</span> <span class="b">lock</span><span class="f">-&gt;</span><span class="b">lock</span> <span class="f">&gt;</span> <span class="c">0</span><span class="f">;</span>
<a name="212" /><span class="True">     212:</span> <span class="f">}</span>
<a name="213" /><span class="True">     213:</span> 
<a name="214" /><span class="True">     214:</span> <span class="k">/**</span>
<a name="215" /><span class="True">     215:</span> <span class="k"> * write_can_lock - would write_trylock() succeed?</span>
<a name="216" /><span class="True">     216:</span> <span class="k"> * @lock: the rwlock in question.</span>
<a name="217" /><span class="True">     217:</span> <span class="k"> */</span>
<a name="218" /><span class="True">     218:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">int</span> <span class="b">arch_write_can_lock</span><span class="f">(</span><span class="b">arch_rwlock_t</span> <span class="f">*</span><span class="b">lock</span><span class="f">)</span>
<a name="219" /><span class="True">     219:</span> <span class="f">{</span>
<a name="220" /><span class="True">     220:</span>     <span class="m">return</span> <span class="b">lock</span><span class="f">-&gt;</span><span class="b">write</span> <span class="f">==</span> <a href="cpu.c_macros_ref.html#_V1JJVEVfTE9DS19DTVBfMA__"><span class="b">WRITE_LOCK_CMP</span></a><span class="f">;</span>
<a name="221" /><span class="True">     221:</span> <span class="f">}</span>
<a name="222" /><span class="True">     222:</span> 
<a name="223" /><span class="True">     223:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">void</span> <span class="b">arch_read_lock</span><span class="f">(</span><span class="b">arch_rwlock_t</span> <span class="f">*</span><span class="b">rw</span><span class="f">)</span>
<a name="224" /><span class="True">     224:</span> <span class="f">{</span>
<a name="225" /><span class="True">     225:</span>     <span class="m">asm</span> <span class="m">volatile</span><span class="f">(</span><a href="cpu.c_macros_ref.html#_TE9DS19QUkVGSVhfMA__"><span class="b">LOCK_PREFIX</span></a> <a href="cpu.c_macros_ref.html#_UkVBRF9MT0NLX1NJWkVfMA__"><span class="b">READ_LOCK_SIZE</span></a><span class="f">(</span><span class="b">dec</span><span class="f">)</span> <span class="e">&quot; (%0)\n\t&quot;</span>
<a name="226" /><span class="True">     226:</span>              <span class="e">&quot;jns 1f\n&quot;</span>
<a name="227" /><span class="True">     227:</span>              <span class="e">&quot;call __read_lock_failed\n\t&quot;</span>
<a name="228" /><span class="True">     228:</span>              <span class="e">&quot;1:\n&quot;</span>
<a name="229" /><span class="True">     229:</span>              <span class="f">::</span><a href="cpu.c_macros_ref.html#_TE9DS19QVFJfUkVHXzA_"><span class="b">LOCK_PTR_REG</span></a> <span class="f">(</span><span class="b">rw</span><span class="f">)</span> <span class="f">:</span> <span class="e">&quot;memory&quot;</span><span class="f">)</span><span class="f">;</span>
<a name="230" /><span class="True">     230:</span> <span class="f">}</span>
<a name="231" /><span class="True">     231:</span> 
<a name="232" /><span class="True">     232:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">void</span> <span class="b">arch_write_lock</span><span class="f">(</span><span class="b">arch_rwlock_t</span> <span class="f">*</span><span class="b">rw</span><span class="f">)</span>
<a name="233" /><span class="True">     233:</span> <span class="f">{</span>
<a name="234" /><span class="True">     234:</span>     <span class="m">asm</span> <span class="m">volatile</span><span class="f">(</span><a href="cpu.c_macros_ref.html#_TE9DS19QUkVGSVhfMA__"><span class="b">LOCK_PREFIX</span></a> <a href="cpu.c_macros_ref.html#_V1JJVEVfTE9DS19TVUJfMA__"><span class="b">WRITE_LOCK_SUB</span></a><span class="f">(</span><span class="f">%</span><span class="c">1</span><span class="f">)</span> <span class="e">&quot;(%0)\n\t&quot;</span>
<a name="235" /><span class="True">     235:</span>              <span class="e">&quot;jz 1f\n&quot;</span>
<a name="236" /><span class="True">     236:</span>              <span class="e">&quot;call __write_lock_failed\n\t&quot;</span>
<a name="237" /><span class="True">     237:</span>              <span class="e">&quot;1:\n&quot;</span>
<a name="238" /><span class="True">     238:</span>              <span class="f">::</span><a href="cpu.c_macros_ref.html#_TE9DS19QVFJfUkVHXzA_"><span class="b">LOCK_PTR_REG</span></a> <span class="f">(</span><span class="f">&amp;</span><span class="b">rw</span><span class="f">-&gt;</span><span class="b">write</span><span class="f">)</span><span class="f">,</span> <span class="e">&quot;i&quot;</span> <span class="f">(</span><a href="cpu.c_macros_ref.html#_UldfTE9DS19CSUFTXzA_"><span class="b">RW_LOCK_BIAS</span></a><span class="f">)</span>
<a name="239" /><span class="True">     239:</span>              <span class="f">:</span> <span class="e">&quot;memory&quot;</span><span class="f">)</span><span class="f">;</span>
<a name="240" /><span class="True">     240:</span> <span class="f">}</span>
<a name="241" /><span class="True">     241:</span> 
<a name="242" /><span class="True">     242:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">int</span> <span class="b">arch_read_trylock</span><span class="f">(</span><span class="b">arch_rwlock_t</span> <span class="f">*</span><span class="b">lock</span><span class="f">)</span>
<a name="243" /><span class="True">     243:</span> <span class="f">{</span>
<a name="244" /><span class="True">     244:</span>     <a href="cpu.c_macros_ref.html#_UkVBRF9MT0NLX0FUT01JQ18w"><span class="b">READ_LOCK_ATOMIC</span></a><span class="f">(</span><span class="b">t</span><span class="f">)</span> <span class="f">*</span><span class="b">count</span> <span class="f">=</span> <span class="f">(</span><a href="cpu.c_macros_ref.html#_UkVBRF9MT0NLX0FUT01JQ18w"><span class="b">READ_LOCK_ATOMIC</span></a><span class="f">(</span><span class="b">t</span><span class="f">)</span> <span class="f">*</span><span class="f">)</span><span class="b">lock</span><span class="f">;</span>
<a name="245" /><span class="True">     245:</span> 
<a name="246" /><span class="True">     246:</span>     <span class="m">if</span> <span class="f">(</span><a href="cpu.c_macros_ref.html#_UkVBRF9MT0NLX0FUT01JQ18w"><span class="b">READ_LOCK_ATOMIC</span></a><span class="f">(</span><span class="b">dec_return</span><span class="f">)</span><span class="f">(</span><span class="b">count</span><span class="f">)</span> <span class="f">&gt;=</span> <span class="c">0</span><span class="f">)</span>
<a name="247" /><span class="True">     247:</span>         <span class="m">return</span> <span class="c">1</span><span class="f">;</span>
<a name="248" /><span class="True">     248:</span>     <a href="cpu.c_macros_ref.html#_UkVBRF9MT0NLX0FUT01JQ18w"><span class="b">READ_LOCK_ATOMIC</span></a><span class="f">(</span><span class="b">inc</span><span class="f">)</span><span class="f">(</span><span class="b">count</span><span class="f">)</span><span class="f">;</span>
<a name="249" /><span class="True">     249:</span>     <span class="m">return</span> <span class="c">0</span><span class="f">;</span>
<a name="250" /><span class="True">     250:</span> <span class="f">}</span>
<a name="251" /><span class="True">     251:</span> 
<a name="252" /><span class="True">     252:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">int</span> <span class="b">arch_write_trylock</span><span class="f">(</span><span class="b">arch_rwlock_t</span> <span class="f">*</span><span class="b">lock</span><span class="f">)</span>
<a name="253" /><span class="True">     253:</span> <span class="f">{</span>
<a name="254" /><span class="True">     254:</span>     <span class="b">atomic_t</span> <span class="f">*</span><span class="b">count</span> <span class="f">=</span> <span class="f">(</span><span class="b">atomic_t</span> <span class="f">*</span><span class="f">)</span><span class="f">&amp;</span><span class="b">lock</span><span class="f">-&gt;</span><span class="b">write</span><span class="f">;</span>
<a name="255" /><span class="True">     255:</span> 
<a name="256" /><span class="True">     256:</span>     <span class="m">if</span> <span class="f">(</span><span class="b">atomic_sub_and_test</span><span class="f">(</span><a href="cpu.c_macros_ref.html#_V1JJVEVfTE9DS19DTVBfMA__"><span class="b">WRITE_LOCK_CMP</span></a><span class="f">,</span> <span class="b">count</span><span class="f">)</span><span class="f">)</span>
<a name="257" /><span class="True">     257:</span>         <span class="m">return</span> <span class="c">1</span><span class="f">;</span>
<a name="258" /><span class="True">     258:</span>     <span class="b">atomic_add</span><span class="f">(</span><a href="cpu.c_macros_ref.html#_V1JJVEVfTE9DS19DTVBfMA__"><span class="b">WRITE_LOCK_CMP</span></a><span class="f">,</span> <span class="b">count</span><span class="f">)</span><span class="f">;</span>
<a name="259" /><span class="True">     259:</span>     <span class="m">return</span> <span class="c">0</span><span class="f">;</span>
<a name="260" /><span class="True">     260:</span> <span class="f">}</span>
<a name="261" /><span class="True">     261:</span> 
<a name="262" /><span class="True">     262:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">void</span> <span class="b">arch_read_unlock</span><span class="f">(</span><span class="b">arch_rwlock_t</span> <span class="f">*</span><span class="b">rw</span><span class="f">)</span>
<a name="263" /><span class="True">     263:</span> <span class="f">{</span>
<a name="264" /><span class="True">     264:</span>     <span class="m">asm</span> <span class="m">volatile</span><span class="f">(</span><a href="cpu.c_macros_ref.html#_TE9DS19QUkVGSVhfMA__"><span class="b">LOCK_PREFIX</span></a> <a href="cpu.c_macros_ref.html#_UkVBRF9MT0NLX1NJWkVfMA__"><span class="b">READ_LOCK_SIZE</span></a><span class="f">(</span><span class="b">inc</span><span class="f">)</span> <span class="e">&quot; %0&quot;</span>
<a name="265" /><span class="True">     265:</span>              <span class="f">:</span><span class="e">&quot;+m&quot;</span> <span class="f">(</span><span class="b">rw</span><span class="f">-&gt;</span><span class="b">lock</span><span class="f">)</span> <span class="f">:</span> <span class="f">:</span> <span class="e">&quot;memory&quot;</span><span class="f">)</span><span class="f">;</span>
<a name="266" /><span class="True">     266:</span> <span class="f">}</span>
<a name="267" /><span class="True">     267:</span> 
<a name="268" /><span class="True">     268:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">void</span> <span class="b">arch_write_unlock</span><span class="f">(</span><span class="b">arch_rwlock_t</span> <span class="f">*</span><span class="b">rw</span><span class="f">)</span>
<a name="269" /><span class="True">     269:</span> <span class="f">{</span>
<a name="270" /><span class="True">     270:</span>     <span class="m">asm</span> <span class="m">volatile</span><span class="f">(</span><a href="cpu.c_macros_ref.html#_TE9DS19QUkVGSVhfMA__"><span class="b">LOCK_PREFIX</span></a> <a href="cpu.c_macros_ref.html#_V1JJVEVfTE9DS19BRERfMA__"><span class="b">WRITE_LOCK_ADD</span></a><span class="f">(</span><span class="f">%</span><span class="c">1</span><span class="f">)</span> <span class="e">&quot;%0&quot;</span>
<a name="271" /><span class="True">     271:</span>              <span class="f">:</span> <span class="e">&quot;+m&quot;</span> <span class="f">(</span><span class="b">rw</span><span class="f">-&gt;</span><span class="b">write</span><span class="f">)</span> <span class="f">:</span> <span class="e">&quot;i&quot;</span> <span class="f">(</span><a href="cpu.c_macros_ref.html#_UldfTE9DS19CSUFTXzA_"><span class="b">RW_LOCK_BIAS</span></a><span class="f">)</span> <span class="f">:</span> <span class="e">&quot;memory&quot;</span><span class="f">)</span><span class="f">;</span>
<a name="272" /><span class="True">     272:</span> <span class="f">}</span>
<a name="273" /><span class="True">     273:</span> 
<a name="274" /><span class="True">     274:</span> <span class="f">#</span><span class="n">define</span> <a href="cpu.c_macros_ref.html#_YXJjaF9yZWFkX2xvY2tfZmxhZ3NfMA__"><span class="b">arch_read_lock_flags</span></a><span class="f">(</span><span class="b">lock</span><span class="f">,</span> <span class="b">flags</span><span class="f">)</span> <span class="b">arch_read_lock</span><span class="f">(</span><span class="b">lock</span><span class="f">)</span>
<a name="275" /><span class="True">     275:</span> <span class="f">#</span><span class="n">define</span> <a href="cpu.c_macros_ref.html#_YXJjaF93cml0ZV9sb2NrX2ZsYWdzXzA_"><span class="b">arch_write_lock_flags</span></a><span class="f">(</span><span class="b">lock</span><span class="f">,</span> <span class="b">flags</span><span class="f">)</span> <span class="b">arch_write_lock</span><span class="f">(</span><span class="b">lock</span><span class="f">)</span>
<a name="276" /><span class="True">     276:</span> 
<a name="277" /><span class="True">     277:</span> <span class="f">#</span><span class="n">undef</span> <a href="cpu.c_macros_ref.html#_UkVBRF9MT0NLX1NJWkVfMA__"><span class="b">READ_LOCK_SIZE</span></a>
<a name="278" /><span class="True">     278:</span> <span class="f">#</span><span class="n">undef</span> <a href="cpu.c_macros_ref.html#_UkVBRF9MT0NLX0FUT01JQ18w"><span class="b">READ_LOCK_ATOMIC</span></a>
<a name="279" /><span class="True">     279:</span> <span class="f">#</span><span class="n">undef</span> <a href="cpu.c_macros_ref.html#_V1JJVEVfTE9DS19BRERfMA__"><span class="b">WRITE_LOCK_ADD</span></a>
<a name="280" /><span class="True">     280:</span> <span class="f">#</span><span class="n">undef</span> <a href="cpu.c_macros_ref.html#_V1JJVEVfTE9DS19TVUJfMA__"><span class="b">WRITE_LOCK_SUB</span></a>
<a name="281" /><span class="True">     281:</span> <span class="f">#</span><span class="n">undef</span> <a href="cpu.c_macros_ref.html#_V1JJVEVfTE9DS19DTVBfMA__"><span class="b">WRITE_LOCK_CMP</span></a>
<a name="282" /><span class="True">     282:</span> 
<a name="283" /><span class="True">     283:</span> <span class="f">#</span><span class="n">define</span> <a href="cpu.c_macros_noref.html#_YXJjaF9zcGluX3JlbGF4XzA_"><span class="b">arch_spin_relax</span></a><span class="f">(</span><span class="b">lock</span><span class="f">)</span>    <span class="b">cpu_relax</span><span class="f">(</span><span class="f">)</span>
<a name="284" /><span class="True">     284:</span> <span class="f">#</span><span class="n">define</span> <a href="cpu.c_macros_noref.html#_YXJjaF9yZWFkX3JlbGF4XzA_"><span class="b">arch_read_relax</span></a><span class="f">(</span><span class="b">lock</span><span class="f">)</span>    <span class="b">cpu_relax</span><span class="f">(</span><span class="f">)</span>
<a name="285" /><span class="True">     285:</span> <span class="f">#</span><span class="n">define</span> <a href="cpu.c_macros_noref.html#_YXJjaF93cml0ZV9yZWxheF8w"><span class="b">arch_write_relax</span></a><span class="f">(</span><span class="b">lock</span><span class="f">)</span>    <span class="b">cpu_relax</span><span class="f">(</span><span class="f">)</span>
<a name="286" /><span class="True">     286:</span> 
<a name="287" /><span class="True">     287:</span> <span class="f">#</span><span class="n">endif</span> <span class="k">/* _ASM_X86_SPINLOCK_H */</span>
<a name="288" /><span class="True">     288:</span> </pre>
  </body>
</html>
